
import sys
import time
import random
import re

from Config import *
from SpiderUtil import *
from ProxyUtil import *
from RabbitUtil import RabbitUtil
from SeleniumUtil import SeleniumUtil
from SeleniumUtil import CheckDriverExit


#selenium
from selenium import webdriver
from selenium.webdriver import ChromeOptions
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.keys import Keys
#反爬
import undetected_chromedriver as uc
#BeautifulSoup
from bs4 import BeautifulSoup

from ErrorCode import *

#多线程处理
from threading import Thread
#线程池相关
from concurrent.futures import ThreadPoolExecutor, wait, as_completed

#----------------------------------------解析网页 一个列表页面也就是48个详情----------------------------------------
#解析网页中拍卖房产信息列表 "./html/SFList.html"
def SFListParse(htmlcontent, PropName, Province, City):
    #保存一个临时文件
    # with open(Config["Spider"]["workDir"] + "/Temp.html", "wb") as f:
    #     f.write(htmlcontent);

    #开始解析
    soup = BeautifulSoup(htmlcontent, "lxml");

    #解析出来的信息
    SFInfoList = [];

    #获得拍卖房产信息
    SFList = soup.select('div[class="sf-item-list"] li');

    #逐个信息解析
    for Idx,SF in enumerate(SFList):
        try:
            #详细信息页链接 //sf-item.taobao.com/sf_item/712418425380.htm?track_id=405a29ac-7e74-4283-8675-59b884f033ea
            href = SF.select_one("a")['href'];
            href = re.findall(r"^(.*)\?", href)[0];
            code = re.findall(r"(\d+)\.htm", href)[0];

            #-------------------header-section
            #缩略图地址
            imginfos = SF.select_one('a > div.header-section > img');
            img = imginfos['src'];
            #如果是特殊页面，需要进一步获取
            if img=="//g.alicdn.com/s.gif":
                img = imginfos['data-ks-lazyload'];

            #房屋名称
            title = SF.select_one('a > div.header-section > p').text.strip();
            #-------------------info-section
            #如果拍卖已结束，不再进行解析
            #开始时间
            StartTimeStr = SF.select_one('a > div.info-section > p.time.time-todo > span.value').text;
            #预计结束时间
            AboutEndTimeStr = SF.select_one('a > div.info-section > p.time.time-doing > span.value').text;
            #结束时间
            EndTimeStr   = SF.select_one('a > div.info-section > p.time.time-done > span.value').text;

            #价格等信息
            PriceInfo1 = re.sub("[\s]{1,}", ' ', SF.select_one('a > div.info-section > p.price.price-todo.pai-status-todo-show').text);
            ItemList = SF.select('a > div.info-section > p.price.price-assess');
            if len(ItemList)==0:
                PriceInfo2 = "";
            else:
                PriceInfo2 = re.sub("[\s]{1,}", ' ', ItemList[0].text);
            StartTime  = re.sub("[\s]{1,}", ' ', SF.select_one('a > div.info-section > p.time.time-todo.pai-status-todo-show').text);
            #-------------------footer-section
            #关注信息
            GZInfo = re.sub("[\s]{1,}", ' ', SF.select_one('a > div.footer-section').text);

            Status = StatusDictAli(SF["class"][1])
            #输出数据列表
            resultTuple = {
                "houseCode": code,
                "auctionUrl": "https:" + href,
                "banner": "https:" + img,
                "title": title,
                # "startPrice": PriceInfo1.strip(),
                # "estimatePrice": PriceInfo2.strip(),
                # "startTime": StartTimeStr,
                # "aboutEndTime": AboutEndTimeStr,
                # "endTime": EndTimeStr,
                "houseType": PropName,
                "province": Province,
                "city": City,
                "status": Status,
                "origin": '阿里法拍',
            }
            # resultTuple = (code, "https:"+href, "https:"+img, title, PriceInfo1.strip(), PriceInfo2.strip(), StartTime.strip(), GZInfo.strip(), StartTimeStr, AboutEndTimeStr, EndTimeStr, PropName);
            SFInfoList.append(resultTuple);
            #show for debug
            # print("-"*10, Idx, resultTuple);
            #end for
        except Exception as e:
            print("list parse except info ", e)
        
    #end
    return SFInfoList;
    #end


#根据地市信息，获得各地市资产信息列表
def GetSFList(AreaInfo, City, CurrPage, HouseType, driver):

    #延时等待浏览器打开完成
    wait = WebDriverWait(driver, 2);

    #循环处理
    while True:
        try:
            CheckDriverExit(driver)

            AliSFURL = "https://sf.taobao.com/item_list.htm";
            #打开资产列表页面
            driver.get(AliSFURL);
            time.sleep(2);

            #如果未进入目标页，则触发异常，等待人工处理
            while driver.current_url != AliSFURL:
                print("Not LoginURL Wait!! LoginURL=", AliSFURL);
                WarningPrompt(Type=EList["E101"]);
                time.sleep(5);
            
            #等待页面加载
            time.sleep(2);
            
            #点击对应省份、地市、资产类别，分别进行列表的获取
            while True:
                try:
                    #如果没有省的信息，则无需处理
                    if AreaInfo[0] == None:
                        print("There is no province.. ", AreaInfo);
                        break;
                    
                    #每个列表页解析的页数  默认所有页面都爬取
                    ListPages = Config["Spider"]["maxPage"];
                    #省份名称
                    provPage = AreaInfo[0].split("?");
                    ProvName = provPage[0];
                    if len(provPage) > 1:
                        ListPages = int(provPage[1])

                    
                    #显示处理的地区
                    print("processing..... ", ProvName);
                    
                    #寻找并点击省份
                    try:
                        #标的物所在地
                        LocationElem = driver.find_element(By.CSS_SELECTOR, 'ul[class*="sf-filter-row-location"]');
                        #省份标签
                        ProvElem = LocationElem.find_element(By.PARTIAL_LINK_TEXT, ProvName);
                        #点击省
                        if ProvElem.is_enabled() and ProvElem.is_displayed():
                            ProvElem.click();
                        #不可点击
                        else:
                            print("Province is not clickable!!!", ProvName);
                            continue;
                    
                        #等待网页加载
                        time.sleep(3);

                        #确认省份已经被选中
                        ProvElem = driver.find_element(By.CSS_SELECTOR, 'ul.sf-filter-row-location ul.condition > li.unfold > em > a');
                        if not ProvElem.text.startswith(ProvName):
                            print("ProvElem is not be selected!!");
                            continue;

                    except:
                        print("Press Province Error!!!")
                        WarningPrompt(Type=EList["E102"]);
                        #检查浏览器是否可用
                        CheckDriverExit(driver)
                        time.sleep(5)
                        continue;
                    
                    
                    #------------------------点击地市------------------------
                    #获得地市列表
                    CityElems = driver.find_elements(By.CSS_SELECTOR, 'div[class*="J_SubCondition"] > ul > li');
                    # CityElems = driver.find_elements(By.CSS_SELECTOR, f'div[class*="J_SubCondition"] li.J_FilterCity > em > a');
                    if len(CityElems)==0:
                        print("There is no CityTag!!!");
                        continue;

                    #遍历每个地市，并分别点击
                    for CityIdx in range(len(CityElems)):
                        #地市处理异常时，不断重试本地市
                        Coder = "utf8";
                        while True:
                            try:
                                CityElems = driver.find_elements(By.CSS_SELECTOR, f'div[class*="J_SubCondition"] li.J_FilterCity > em > a');
                                #本次拟点击地市
                                CityElem = CityElems[CityIdx];
                                #城市名称
                                CityName = CityElem.text.split()[0];

                                # 直接到指定城市
                                if City != None and not City.startswith(CityName):
                                    break;
                                # 匹配到一致的了改为None  后面就不判断了
                                City = None;
                                    
                                #点击城市
                                if CityElem.is_enabled() and CityElem.is_displayed():
                                    CityElem.click();
                                #地市不可点击
                                else:
                                    print("City is not clickable!!!", CityElem.text);
                                    continue;
                                
                                #等待页面加载
                                time.sleep(2);

                                #确认被点击的城市
                                CheckCityElem = driver.find_element(By.CSS_SELECTOR, 'ul.sf-filter-row-location div.J_SubCondition li.J_FilterCity a.selected');
                                if not CheckCityElem.text.startswith(CityName):
                                    print("CityElem is not be selected!!");
                                    continue;

                                #-------------------------------------遍历每个地市下的4种资产-------------------------------------
                                houseTypeArr = ["住宅用房", "商业用房", "工业用房", "其他用房"];
                                #找到过滤条件参数块
                                for i in range(len(houseTypeArr)):
                                    while True:
                                        PropElems = driver.find_elements(By.CSS_SELECTOR, f"div.sf-filter ul.sf-filter-row:nth-child(1) ul.condition a");
                                        PropElem = PropElems[i];
                                        #资产名称
                                        PropName = PropElem.text;
                                        # 直接到指定类型
                                        if HouseType != None and not HouseType.startswith(PropName):
                                            break;
                                        # 匹配到一致的了改为None  后面就不判断了
                                        HouseType = None;

                                        #名称在我们需要的范围内
                                        if PropName in houseTypeArr\
                                            and PropElem.is_enabled() and PropElem.is_displayed:
                                            PropElem.click();
                                        #不可点击，就不处理
                                        else:
                                            continue;
                                        
                                        #等待页面加载
                                        time.sleep(2);
                                        
                                        #校验类型点击成功
                                        PropElem = driver.find_element(By.CSS_SELECTOR, 'div.sf-filter ul.sf-filter-row:nth-child(1) ul.condition > li > em > a.selected');
                                        if not PropElem.text.startswith(PropName):
                                            print("PropElem is not be selected!!");
                                            #刷新页面
                                            driver.refresh();
                                            time.sleep(5);
                                            continue;
                                        

                                        # 类型点击成功
                                        #等待网页加载
                                        time.sleep(random.choice(range(10,30)));

                                        # 添加分页参数 直接打到指定页码
                                        if CurrPage != None:
                                            driver.get(driver.current_url+'&page='+CurrPage);
                                        CurrPage = None;
                                        #--------------------------------------------------------------------------------------------
                                        #最多访问前5页
                                        for i in range(0, ListPages):
                                            try:
                                                #网页Content
                                                #time.sleep(5);
                                                htmlcontent = driver.page_source.encode(Coder, "ignore");

                                                #将网页进行解析，获得房屋信息列表
                                                SFList = SFListParse(htmlcontent, PropName, ProvName, CityName);
                                                
                                                PageNum = 0;
                                                #将解析获得的房屋信息，保存成JSON文件
                                                if len(SFList):
                                                    CurrentElem = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.CSS_SELECTOR, "div.J_Pagination span.current")));
                                                    if CurrentElem.text != None:
                                                        PageNum = int(CurrentElem.text)

                                                    # 数据存到数据库 调用接口
                                                    PageListSave({"grabProvince": ProvName, "grabCity": CityName, "houseType": PropName, "grabPage": str(PageNum), "spiderHouses": SFList});
                                                    # 状态修改为进行中
                                                    StatusModify(Status=3)
                                                    print("Ali List Success, Prov="+ProvName+",City="+CityName+",Type="+PropName+",Page="+str(PageNum))
                                                #结束访问，无需切换下一页
                                                if i>=(ListPages-1):
                                                    break;
                                                
                                                #点击下一页
                                                StopProcess = 0;
                                                while True:
                                                    try:
                                                        if PageNum == 0:
                                                            # 主动获取到当前的页码
                                                            CurrentElem = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.CSS_SELECTOR, "div.J_Pagination span.current")));
                                                            if CurrentElem.text != None:
                                                                PageNum = int(CurrentElem.text)
                                                        
                                                        if PageNum >= ListPages:
                                                            StopProcess = 1;
                                                            break;

                                                        #NextElem = driver.find_element(By.CSS_SELECTOR, "div.J_Pagination a.next");
                                                        NextElem = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.CSS_SELECTOR, "div.J_Pagination a.next")));
                                                        if NextElem.is_enabled() and NextElem.is_displayed:
                                                            NextElem.click();
                                                            break;
                                                        else:
                                                            break;
                                                    #异常 无法解析下一页按钮，则停止处理本Area
                                                    except Exception as e:
                                                        print("Can't click Next!!", e);
                                                        # WarningPrompt(Type=5);
                                                        StopProcess = 1;
                                                        break;
                                                
                                                #如果可以结束5个页的解析，则直接退出
                                                if StopProcess==1:
                                                    break;
                                                #
                                                time.sleep(random.choice(range(10,30)));
                                                #end for
                                            except Exception as e:
                                                print("Parse Page Item error!! ", e);
                                                WarningPrompt(Type=EList["E106"]);
                                                CheckDriverExit(driver)
                                        #end for PageNum
                                        time.sleep(random.choice(range(10,30)));
                                        break;
                                    #end while True 点击资产过滤条件

                                #end for Property
                                break;

                            except Exception as e:
                                print("Parse Page error!! ", e);
                                WarningPrompt(Type=EList["E103"]);
                                #检查浏览器是否可用
                                CheckDriverExit(driver)
                                time.sleep(5);
                                continue;
                    break;
                except:
                    print("Parse City error!! ", e);
                    WarningPrompt(Type=EList["E104"]);
                    #检查浏览器是否可用
                    CheckDriverExit(driver)
                    time.sleep(3);
                    continue;
                
            #end while 省份、地址、资产类别遍历
            break;

        except Exception as e:
            print("Parse List Error!!!!", e);
            WarningPrompt(Type=EList["E105"]);
            #检查浏览器是否可用
            CheckDriverExit(driver)
            continue;
    #end while True
    #end


def ListParseOneProv(Prov, City, CurrPage, HouseType, driver):
    #获得各地市的房产列表页，并插入到数据库
    AreaInfo = (Prov, None);
    GetSFList(AreaInfo, City, CurrPage, HouseType, driver);

# 处理省份函数
def ProvConsumer(Prov, City, CurrPage, HouseType, driver, SeleniumH):
    try:
        print("Consume Msg :", Prov);
        #如果浏览器关闭，则触发异常直接结束任务
        if SeleniumH.Check()==False:
            SeleniumH.Quit();
            sys.exit(1);
        #
        ListParseOneProv(Prov, City, CurrPage, HouseType, driver);
        #输出结果
        return 0;
    except Exception as e:
        print("queue QueueProv except info ", e)
    
# 省份解析
def ProvParse(proxyUrl, City, CurrPage, HouseType, serverIp):
    RabbitParam = Config["RabbitMQ"];
    #初始化Rabbit
    RabbitH = RabbitUtil(host=RabbitParam["host"], user=RabbitParam["user"], passwd=RabbitParam["passwd"], vhost=RabbitParam["vhost"]);
    #Selenium类 初始化
    SeleniumH = SeleniumUtil(ProxyInfo=PorxyInfo(proxyUrl)); #ProxyInfo=PorxyInfo());
    driver = SeleniumH.GetDriver();

    #等待浏览器打开
    time.sleep(2);
    # 消费队列
    def ProvQueue(Prov):
        return ProvConsumer(Prov.decode('utf-8'), City, CurrPage, HouseType, driver, SeleniumH)
    
    serverIp = serverIp.replace(".", "_");
    QueueName = Config["Spider"]["alilqName"]+serverIp
    RabbitH.CreateQueues([QueueName]);
    RabbitH.RegistConsumer(QueueName, ProvQueue);
    #释放资源
    SeleniumH.Quit();
    RabbitH.Close();


# 执行列表信息获取
if __name__ == "__main__":
    # 获取爬虫基本参数
    ParaMap = GetSpiderParams()
    threadNum = ParaMap["threadNum"]

    #创建2个线程的线程池
    with ThreadPoolExecutor(max_workers=threadNum) as t:
        #创建待运行线程（如果中途线程中，关掉浏览器，则直接结束任务，并触发新的任务执行）
        all_task = [ t.submit(ProvParse, ParaMap['proxyUrl'], ParaMap['grabCity'], ParaMap['grabPage'], ParaMap['houseType'], ParaMap['serverIp']) for _ in range(threadNum*20) ];
        #print(wait(all_task, timeout=2.5));
        #等待所有任务完成
        for future in as_completed(all_task):
            data = future.result();
            print("in main: get page {}s success".format(data));

    sys.exit(0);





